const fs = require('fs');

const superagent = require('superagent');
const cheerio = require('cheerio');
const Eventproxy = require('eventproxy');
const async = require('async');
const ep = new Eventproxy(); //事件代理
const mock = require('mockjs'); //移入数据模拟模块


let reptileUtil = {
    isMove: false,
    queue: [], //爬取队列
    push(urls, event) { //进入队列
        if (!this.isMove) { //不在抓取，则进入抓取
            this.move(urls, event);
        } else {
            this.queue.push({ //抓取中，进入队列等待
                urls,
                event
            });
        }
    },
    move(urls, event) {
        this.isMove = true;
        async.mapLimit(urls, 5, async (url) => {
            let delay = Math.floor(Math.random() * 300000000) % 1000;
            console.log('正在抓取的是', url, '，耗时' + delay + '毫秒');
            try {
                let pres = await superagent.get(url); //发起请求
                return cheerio.load(pres.text, {
                    decodeEntities: false
                }); //使用cheerio加载pres.text 生成类jq接口
            } catch (err) {
                console.log(err);
            }
        }, (err, results) => {
            if (err) throw err;
            ep.emit(event, results); //结束后发送结果
            if (this.queue.length) { //出队
                const {
                    urls,
                    event
                } = this.queue.shift();
                this.move(urls, event);
            } else {
                this.isMove = false;
            }
        })
    }
}


reptileUtil.push(['http://news.baidu.com'], 'entryEnd'); //爬取入口

// 监听爬取入口
ep.on('entryEnd', $s => {
    const $ = $s[0]; //门户
    const $category = $('.menu-list').eq(1).find('ul li a');
    const categoryHrefs = [],
        category = [];

    for (let i = 0, len = $category.length; i < len; i++) {
        let $temp = $category.eq(i);
        categoryHrefs.push('http://news.baidu.com' + $temp.attr('href'));
        category.push({
            id: i == 0 ? 999 : i,
            name: $temp.text(),
            is_top: 1
        });
    }

    // 分类写入category
    fs.writeFileSync('../data/category.json', JSON.stringify(category), 'utf-8', err => {
        if (err) throw err;
    });

    reptileUtil.push(categoryHrefs, 'categoryEnd'); //爬取分类页
});


// 监听分类页爬取
ep.on('categoryEnd', $s => {
    //详情页链接集合
    const details = $s.map($ => {
        return Array.from($('a')).map(v => {
            if ($(v).attr('href') && $(v).attr('href').indexOf('http://baijiahao.baidu.com/s') > -1) {
                return $(v).attr('href');
            }
            return ''
        }).filter(v => v);
    });

    details.forEach(v => {
        reptileUtil.push(v, 'detailEnd'); //进入爬取队列
    })
});

ep.after('detailEnd', 13, lists => {
    let postId = 0;
    let covers = [];
    let coverId = 0;
    let posts = lists.map((list, categoryId) => {
        return list.map(($, i) => {
            postId++;
            let title = $('.article-title h2').text();
            let create_date = '2020-' + $('.article-source .date').text().replace(/[^\d]*([\d-]+)[^d]*/, '$1') + ' ' + $('.article-source .time').text();
            create_date = new Date(create_date).toLocaleString().replace(/([\d-]+)[^\d-:]*([\d:]+).+?/, '$1T$2.000Z');
            if (!/([\d-]+)[^\d-:]*([\d:]+)/.test(create_date)) return null;

            let type = 1;
            $imgs = $('.article-content img');
            let cover = Array.from($imgs).map(v => {
                coverId++;
                return {
                    url: $(v).attr('src'),
                    id: coverId,
                    postId,
                }
            });
            if (!cover.length) {
                coverId++;
                cover.push({
                    url: 'https://ss0.bdstatic.com/70cFvHSh_Q1YnxGkpoWK1HF6hhy/it/u=2807673150,636733386&fm=26&gp=0.jpg',
                    uid: coverId,
                    id: coverId,
                    postId
                });
            } else if (cover.length < 3) {
                cover.length = 1;
            } else {
                cover.length = 3;
            }
            covers = [...covers, ...cover];

            let content = $('.article-content').html();
            let userId = $('.author-name').text();
            if (!content) return null;
            if (!title) return null;
            if (!create_date) return null;
            return {
                id: postId,
                title,
                create_date,
                type,
                content,
                userId,
                categoryId: categoryId == 0 ? 999 : categoryId
            }
        }).filter(v => v);
    }).reduce((prev, cur) => [...prev, ...cur], []);

    let username = 10000; //用户名
    let start = new Date('2010-01-01').getTime();
    let end = new Date('2016-01-01').getTime();
    let users = Array.from(new Set(posts.map(v => v.userId))).map((v, i) => ({
        id: i + 1,
        nickname: v,
        username: ++username,
        password: 123,
        create_date: new Date(Math.floor(Math.random() * (end - start)) + end).toLocaleString().replace(/([\d-]+)[^\d-:]*([\d:]+).+?/, '$1T$2.000Z'),
        gender: Math.round(Math.random()),
        head_img: ''
    }));

    posts.forEach(v => {
        users.some((item, index) => {
            if (item.nickname == v.userId) {
                v.userId = item.id;
                return;
            }
        })
    })

    let likeId = 0;
    let likes = users.map((user) => {
        return posts.map((post) => {
            return {
                id: ++likeId,
                userId: user.id,
                postId: post.id,
                has_like: Math.round(Math.random() - 0.3),
                has_star: Math.round(Math.random() - 0.3)
            }
        })
    }).reduce((prev, cur) => [...prev, ...cur], []);

    let followId = 0;
    let follow = users.map(user => {
        return users.map(follow => {
            if (user.id == follow.id) return null;
            return {
                id: ++followId,
                userId: user.id,
                followUserId: follow.id,
                has_follow: Math.round(Math.random() - 0.35),
            }
        }).filter(v => v);
    }).reduce((prev, cur) => [...prev, ...cur], []);

    fs.writeFileSync('../data/post.json', JSON.stringify(posts), 'utf-8', err => {
        if (err) throw err;
    });
    fs.writeFileSync('../data/user.json', JSON.stringify(users), 'utf-8', err => {
        if (err) throw err;
    });
    fs.writeFileSync('../data/like.json', JSON.stringify(likes), 'utf-8', err => {
        if (err) throw err;
    });
    fs.writeFileSync('../data/comment.json', JSON.stringify(randomComment(8000)), 'utf-8', err => {
        if (err) throw err;
    });
    fs.writeFileSync('../data/follow.json', JSON.stringify(follow), 'utf-8', err => {
        if (err) throw err;
    });
    fs.writeFileSync('../data/cover.json', JSON.stringify(covers), 'utf-8', err => {
        if (err) throw err;
    });


    function randomComment(num) {
        let data = mock.mock({
            [`Comment|${num}`]: [{ //随机项键名
                //随机id
                'id|+1': 1,
                //随机作者名
                userId: `@NATURAL(1,${users.length})`,
                //内容
                content: '@CSENTENCE(8,32)',
                //文章id
                postId: `@NATURAL(1,${posts.length})`,
            }]
        });
        data.Comment.forEach((v, i) => {
            if (i < 3000) {
                v.parentId = null;
            } else {
                v.parentId = Math.round(Math.random()) ? null : Math.floor(Math.random() * 2999) + 1;
            }
            let start = new Date('2016-01-01').getTime();
            let end = new Date('2020-01-01').getTime();
            v.create_date = new Date(Math.floor(Math.random() * (end - start)) + end).toLocaleString().replace(/([\d-]+)[^\d-:]*([\d:]+).+?/, '$1T$2.000Z');
        })

        return data.Comment;
    }
});